#include "DictProducer.h"
#include "SplicTool1.h"
#include<iostream>
#include <sstream>
#include <sys/types.h>
#include <dirent.h>
#include <fstream>
using namespace std;
class WordSegmentation;

DictProducer::DictProducer(const string&dir,WordSegmentation&splitTool)
:_dir(dir)
,_splitTool(splitTool)
{
      _files.reserve(100);
}



void DictProducer::buildCnDict() {
      getFiles();// 获取文件路径
    for (auto& file : _files) {
        ifstream ifs(file, ios::binary | ios::ate);  // 以二进制模式打开文件，并将文件指针定位到文件末尾
        if (!ifs.good()) {
            cerr << "Failed to open file: " << file << endl;
            return;
        }
        cout << file << endl;  // 打印当前文件路径

        // 获取文件大小
        size_t length = ifs.tellg();

        // 分配足够大的缓冲区来存储文件内容
        char* buff = new char[length+1];

        // 将文件指针重新定位到文件开头并读取文件内容到缓冲区
        ifs.seekg(0, ios::beg);
        ifs.read(buff, length+1);

        string txt(buff, length);

        delete[] buff;  // 释放内存
        
        vector<string> tmp = _splitTool(txt);  // 使用分词工具对文本进行分词
        for (auto& i : tmp) {
            // 如果是中文且不在停用词集中出现过
            if (!_stopWord.count(i) && getByteNum_UTF8(i[0]) == 3) {
                pushDict(i);  // 存储单词到词典中
            }
        }

        ifs.close();  // 关闭文件
    }
}


void DictProducer::pushDict(const string&word){
        ++_dict[word];
}




void DictProducer::buildEnDict(){
    getFiles();
    for(auto& file:_files){
           ifstream ifs(file);
           if(!ifs){
               cerr<<"ifstream open file"<<file<<"error"<<endl;
               return ;
           }
       //读取一行数据
        string line;

        while(getline(ifs,line)){
            //处理字符
            preprocessline(line);
             
            istringstream iss(line);
            string word;
            while(iss>>word){
                //如果为空或者在停用词集中出现过就重新开始循环
                 if(word.empty() || _stopWord.count(word) > 0)
                    continue;

                ++_dict[word];

            }
        }
        ifs.close();
    }
}



//对整行数据进行处理，标点符号变为空格，大写字母变为小写字母
void DictProducer::preprocessline(string&line){
    for(auto&ch:line){
        if(!isalpha(ch)){
            ch=' ';
        }
        else if(isupper(ch)){
            ch=tolower(ch);
        }
    }
}



//将词典写入文件
void DictProducer::storeDict(const char*filepath){
      ofstream ofs(filepath);
      if(!ofs)
      {
      cerr<<"ofs open"<<filepath<<"error"<<endl;
      return;
      }

      for(auto&elem:_dict){
          ofs<<elem.first<<" "<<elem.second<<endl;
      }
      ofs.close();
}



void DictProducer::buildIndex()
{
       int i=0;//记录下标
       for(auto elem:_dict){
             string word=elem.first;
             size_t charNums=word.size()/getByteNum_UTF8(word[0]);

             for(size_t idx=0,n=0;n!=charNums;++idx,++n){
                 size_t charLen=getByteNum_UTF8(word[idx]);
                 string subword=word.substr(idx,charLen);

                 _index[subword].insert(i);
                 idx+=(charLen-1);
             }
             ++i;
       }
}

void DictProducer::outputIndex(const string&filepath){
        ofstream ofs(filepath);
        if(!ofs){
        cerr<<"ofs open"<<filepath<<"error"<<endl;
        return ;
        }

        for(auto&it:_index){
            ofs<<it.first<<" ";
            for(auto&index:it.second){
                ofs<<index<<" ";
            }
            ofs<<std::endl;
        }
        ofs.close();
}

size_t  DictProducer::getByteNum_UTF8(const char byte)
{
       int byteNum=0;
       for(size_t i=0;i<6;++i)
       {
           if(byte&(1<<(7-i)))
               ++byteNum;
           else
               break;
       }
       return byteNum==0?1:byteNum;
}


void DictProducer::readstopwords(const string&filepath){
    ifstream ifs(filepath);
    if (!ifs) {
        cerr << "ifstream open file " << filepath << " error" << endl;
        return;
    }

    string word;
    while (ifs >> word) {
        _stopWord.insert(word);
    }

    ifs.close();
}

void DictProducer::getFiles()
{
     DIR*pDir=opendir(_dir.c_str());
     if(!pDir)
     {
         perror("opendir");
         cout<<_dir<<endl;
         return;
     }

     struct dirent*ptr;
     while((ptr=readdir(pDir))!=0){
         if(strcmp(ptr->d_name,".")!=0&&strcmp(ptr->d_name,"..")!=0){
              _files.push_back(_dir+"/"+ptr->d_name);
         }
     }
     closedir(pDir);
}


